package fr.utbm.gi.vi51.slizers.pw.learning;

import java.io.Serializable;
import java.util.Map;
import java.util.TreeMap;

import fr.utbm.gi.vi51.slizers.framework.learning.QAction;
import fr.utbm.gi.vi51.slizers.framework.learning.QComparator;
import fr.utbm.gi.vi51.slizers.framework.learning.QLearner;
import fr.utbm.gi.vi51.slizers.framework.learning.QState;
import fr.utbm.gi.vi51.slizers.framework.learning.QValueStore;
import fr.utbm.gi.vi51.slizers.pw.agent.action.CreatureAction;

public class CreatureQLearner extends QLearner<CreatureQState,CreatureQAction> implements Serializable
{
	/**
	 * Version number associated to the serializable class.
	 */
	private static final long	serialVersionUID	= -2954260195345674634L;
	/**
	 * The tree map storing states and their associated actions/rewards pairs.
	 */
	private final Map< CreatureQState, Map< CreatureQAction, Float > >	rewards	=
		new TreeMap< CreatureQState, Map< CreatureQAction, Float >>( new QComparator() );

	/**
	 * Parameterized constructor.
	 * 
	 * @param _problem the learner's problem
	 */
	public CreatureQLearner( CreatureQProblem _problem )
	{
		super( _problem );
		fillRewardsMap();
	}

	/**
	 * Parameterized constructor.
	 * 
	 * @param _problem the learner's problem
	 * @param _qvaluestore the learner's Q-value store
	 */
	public CreatureQLearner( CreatureQProblem _problem,
			QValueStore< CreatureQState, CreatureQAction > _qvaluestore )
	{
		super( _problem, _qvaluestore );
		
		fillRewardsMap();
	}

	private void fillRewardsMap()
	{
		// TODO: fill the rewards map
		for ( CreatureState currentState : CreatureState.values() )
		{
			Map<CreatureQAction, Float> currentMap = new TreeMap< CreatureQAction, Float >( new QComparator() );
			this.rewards.put(new CreatureQState(currentState), currentMap);
		}
		
		CreatureQState currentState;
		
		// STATE 0 :
		currentState = new CreatureQState(CreatureState.STATE0);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -1.f);

		// STATE 1 :
		currentState = new CreatureQState(CreatureState.STATE1);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.CLIMB_FORWARD), -1.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.UMBRELLA_OPEN), 0.f);
		
		// STATE 2 :
		currentState = new CreatureQState(CreatureState.STATE2);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.2f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.CLIMB_FORWARD), 0.f);

		// STATE 3 :
		currentState = new CreatureQState(CreatureState.STATE3);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.2f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.CLIMB_FORWARD), 0.f);

		// STATE 4 :
		currentState = new CreatureQState(CreatureState.STATE4);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -1.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.NONE), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.UMBRELLA_OPEN), 0.f);

		// STATE 5 :
		currentState = new CreatureQState(CreatureState.STATE5);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -1.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.UMBRELLA_OPEN), 0.f);

		// STATE 6 :
		currentState = new CreatureQState(CreatureState.STATE6);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -1.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), 1.0f);

		// STATE 7 :
		currentState = new CreatureQState(CreatureState.STATE7);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), -1.0f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_BOTTOM), -0.2f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);

		// STATE 8 :
		currentState = new CreatureQState(CreatureState.STATE8);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_BOTTOM), -0.2f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_FORWARD), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.CLIMB_FORWARD), 0.f);

		// STATE 9 :
		currentState = new CreatureQState(CreatureState.STATE9);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_BOTTOM), -0.2f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_FORWARD), 0.f);

		// STATE 10 :
		currentState = new CreatureQState(CreatureState.STATE10);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_BOTTOM), -0.2f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.CLIMB_FORWARD), 0.f);

		// STATE 11 :
		currentState = new CreatureQState(CreatureState.STATE11);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_BOTTOM), -0.2f);

		// STATE 12 :
		currentState = new CreatureQState(CreatureState.STATE12);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_BOTTOM), -0.2f);

		// STATE 13 :
		currentState = new CreatureQState(CreatureState.STATE13);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), -1.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_BOTTOM), -0.2f);

		// STATE 14 :
		currentState = new CreatureQState(CreatureState.STATE14);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), -1.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_BOTTOM), -0.2f);

		// STATE 15 :
		currentState = new CreatureQState(CreatureState.STATE15);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), -0.1f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_BOTTOM), -0.2f);

		// STATE 16 :
		currentState = new CreatureQState(CreatureState.STATE16);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), -1.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);

		// STATE 17 :
		currentState = new CreatureQState(CreatureState.STATE17);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.CLIMB_FORWARD), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_FORWARD), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);

		// STATE 18 :
		currentState = new CreatureQState(CreatureState.STATE18);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_FORWARD), 1.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);

		// STATE 19 :
		currentState = new CreatureQState(CreatureState.STATE19);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.CLIMB_FORWARD), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);

		// STATE 20 :
		currentState = new CreatureQState(CreatureState.STATE20);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);

		// STATE 21 :
		currentState = new CreatureQState(CreatureState.STATE21);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), 0.f);

		// STATE 22 :
		currentState = new CreatureQState(CreatureState.STATE22);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), -1.f);

		// STATE 23 :
		currentState = new CreatureQState(CreatureState.STATE23);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), -1.f);

		// STATE 24 :
		currentState = new CreatureQState(CreatureState.STATE24);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), -0.5f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.MOVE_FORWARD), -0.1f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.TURN_AROUND), 0.f);
		this.rewards.get(currentState).put(new CreatureQAction(CreatureAction.DIG_AND_MOVE_BOTTOM), -0.2f);

		// STATE 24 :
		currentState = new CreatureQState(CreatureState.STATE24);
		for(CreatureAction ca : CreatureAction.values())
			this.rewards.get(currentState).put(new CreatureQAction(ca), 1.f);		
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * fr.utbm.gi.vi51.slizers.framework.learning.QLearner#computeReward(fr.
	 * utbm.gi.vi51.slizers.pw.learning.CreatureQState,
	 * fr.utbm.gi.vi51.slizers.pw.learning.CreatureQAction)
	 */
	public float computeReward( CreatureQState _lastState, CreatureQAction _action )
	{
		CreatureState state = _lastState.getState();
		CreatureAction action = _action.getAction();
		
		return rewards.get( _lastState ).get( _action );
	}
	
	/* (non-Javadoc)
	 * @see fr.utbm.gi.vi51.slizers.framework.learning.QLearner#getQValueStore()
	 */
	public QValueStore< CreatureQState, CreatureQAction > getQValueStore()
	{
		return super.getQValueStore();
	}
	
	
}
